﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.XPath;
using System.IO;
using System.Data;
namespace GateExtractor
{
    class Statistics
    {
        public Dictionary<string, Dictionary<string, int>> tblNessPerSemPerDomain = new Dictionary<string, Dictionary<string, int>>();
        public Dictionary<string, Dictionary<string, int>> tblNessPerSemPerDomainWithoutPolarity = new Dictionary<string, Dictionary<string, int>>();
        public Dictionary<string, Dictionary<string, int>> tblNessGroups = new Dictionary<string, Dictionary<string, int>>();
        public Dictionary<string, Dictionary<string, int>> tblSemsPerFile = new Dictionary<string, Dictionary<string, int>>();
        public Dictionary<string, Dictionary<string, double>> tblSemsPercentsPerFile = new Dictionary<string, Dictionary<string, double>>();
        public Dictionary<string, int> dicSemFreq = new Dictionary<string, int>();
        public Dictionary<string, Dictionary<string, Dictionary<string, int>>> tblAttrsOfInterest = new Dictionary<string, Dictionary<string, Dictionary<string, int>>>();
        public Dictionary<string, Dictionary<string, Dictionary<string, int>>> tblAttrPereriodPerPerNess = new Dictionary<string, Dictionary<string, Dictionary<string, int>>>();
        public Dictionary<string, Dictionary<string, int>> tblSemsPerPeriod = new Dictionary<string, Dictionary<string, int>>();
        public Dictionary<string, List<Double>> tblSemPercentsPerFileOLD = new Dictionary<string, List<double>>();
    }
    class NessAttributes
    {
        //todo
        public string period;
        public string str;
        public string sem;
        public string domain;
        public string strSemPolarity;
        public string filename;

    }

    class Processor
    {
        string[] strPeriods = new string[] { "01", "02", "03", "04", "05", "06" };
        DataTable tblNess = new DataTable();
        List<NessAttributes> listNesses = new List<NessAttributes>();
        List<string> lstSemNames = new List<string>();
        List<string> lstFileNames = new List<string>();
        List<string> lstDomainNames = new List<string>();
        Dictionary<string, int> dicWordsPerFile = new Dictionary<string, int>();
        Dictionary<string, int> dicSemsPerFile = new Dictionary<string, int>();
        double magicMultiplier = 10000;

        //per file stats:
        Statistics statsGlobal = new Statistics();
        //Dictionary<string, Dictionary<string, int>> tblDepsPerSemGlobal = new Dictionary<string, Dictionary<string, int>>();
        //Dictionary<string, Dictionary<string, int>> tblDepsPerSemGlobalWOPol = new Dictionary<string, Dictionary<string, int>>();
        //Dictionary<string, Dictionary<string, int>> tblDepsPerDomGlobal = new Dictionary<string, Dictionary<string, int>>();
        //Dictionary<string, Dictionary<string, int>> tblNessPrefixesGlobal = new Dictionary<string, Dictionary<string, int>>();
        //Dictionary<string, Dictionary<string, int>> tblNessPostfixesGlobal = new Dictionary<string, Dictionary<string, int>>();
        Dictionary<string, Dictionary<string, int>> tblSemsPerFile = new Dictionary<string, Dictionary<string, int>>();

        Dictionary<string, int> dicNessPerFile = new Dictionary<string, int>();
        Dictionary<string, Statistics> dicStatsPerPeriod = new Dictionary<string, Statistics>();
        int numNessesGlobal = 0;
        int cntFiles = 0;
        //Dictionary<string, Dictionary<string, int>> nessGroupsGlobal = new Dictionary<string, Dictionary<string, int>>();

        void AccumulateSimpleIncrement(Dictionary<string, int> dic, string key)
        {
            if (dic.ContainsKey(key))
                dic[key]++;
            else
                dic.Add(key, 1);
        }
        void AccumulateTableIncrement(Dictionary<string, Dictionary<string, int>> table, string rowKey, string colKey)
        {
            if (!table.ContainsKey(rowKey))
                table.Add(rowKey, new Dictionary<string, int>());
            AccumulateSimpleIncrement(table[rowKey], colKey);
        }
        void AccumulateTableOfTablesIncrement(Dictionary<string, Dictionary<string, Dictionary<string, int>>> dataset, string nameTable, string rowKey, string colKey)
        {
            if (!dataset.ContainsKey(nameTable))
                dataset.Add(nameTable, new Dictionary<string, Dictionary<string, int>>());
            AccumulateTableIncrement(dataset[nameTable], rowKey, colKey);
        }

        void DoInitGlobalStats()
        {
            foreach (NessAttributes attr in listNesses)
            {
                if (!lstSemNames.Contains(attr.sem)) lstSemNames.Add(attr.sem); //polarity
                if (!lstDomainNames.Contains(attr.domain)) lstDomainNames.Add(attr.domain);
                if (!lstFileNames.Contains(attr.filename)) lstFileNames.Add(attr.filename);
                if (!dicSemsPerFile.ContainsKey(attr.filename))
                {
                    dicSemsPerFile.Add(attr.filename, 0);
                }
                dicSemsPerFile[attr.filename]++;
            }
            lstSemNames.Sort();
            lstDomainNames.Sort();


            dicWordsPerFile.Add("01_Amelia", 215773);   // blin, v chem razmer-to????
            dicWordsPerFile.Add("01_AP", 73342);
            dicWordsPerFile.Add("01_Clarissa", 1013939);
            dicWordsPerFile.Add("01_MF", 113116);
            dicWordsPerFile.Add("01_Pamela", 525133);
            dicWordsPerFile.Add("01_SH", 15604);
            dicWordsPerFile.Add("02_BL", 182642);
            dicWordsPerFile.Add("02_EV", 156663);
            dicWordsPerFile.Add("02_Marriage", 146880);
            dicWordsPerFile.Add("02_Mary", 23752);
            dicWordsPerFile.Add("02_PP", 122401);
            dicWordsPerFile.Add("02_SS", 120920);
            dicWordsPerFile.Add("03_AG", 69435);
            dicWordsPerFile.Add("03_CR", 70990);
            dicWordsPerFile.Add("03_JE", 186373);
            dicWordsPerFile.Add("03_MM", 323569);
            dicWordsPerFile.Add("03_TMF", 207457);
            dicWordsPerFile.Add("03_WH", 119141);
            dicWordsPerFile.Add("04_AV", 96951);
            dicWordsPerFile.Add("04_DD", 159136);
            dicWordsPerFile.Add("04_FMC", 137920);
            dicWordsPerFile.Add("04_MP", 113202);
            dicWordsPerFile.Add("04_PL", 232115);
            dicWordsPerFile.Add("04_TU", 150859);
            dicWordsPerFile.Add("05_LCL", 117217);
            dicWordsPerFile.Add("05_MD", 64027);
            dicWordsPerFile.Add("05_OR", 80323);
            dicWordsPerFile.Add("05_PR", 57485);
            dicWordsPerFile.Add("05_RE", 154185);
            dicWordsPerFile.Add("05_TJ", 196454);
            dicWordsPerFile.Add("06_BJD", 69838);
            dicWordsPerFile.Add("06_BJE", 96361);
            dicWordsPerFile.Add("06_EA", 62305);
            dicWordsPerFile.Add("06_FLW", 147603);
            dicWordsPerFile.Add("06_PO", 190661);
            dicWordsPerFile.Add("06_SL", 110248);

        }
        void CalcGlobalStatsNewFormat()
        {
            TextWriter wr = new StreamWriter("./out/NewNiceGlobalStats.txt");
            wr.WriteLine("Number of occurancers of word with ness = " + listNesses.Count.ToString());
            List<string> lstUniqueWords = new List<string>();
            List<KeyValuePair<string, string>> lstUniqueNessVariants = new List<KeyValuePair<string, string>>();
            foreach (NessAttributes tokenNessAttrs in listNesses)
            {
                if (!lstUniqueWords.Contains(tokenNessAttrs.str))
                    lstUniqueWords.Add(tokenNessAttrs.str);
                if (!lstUniqueNessVariants.Contains(new KeyValuePair<string, string>(tokenNessAttrs.str, tokenNessAttrs.sem)))
                    lstUniqueNessVariants.Add(new KeyValuePair<string, string>(tokenNessAttrs.str, tokenNessAttrs.sem));
            }
            wr.WriteLine("Number of unique word with ness = " + lstUniqueWords.Count.ToString());
            wr.WriteLine("Number of unique word+sem combinations = " + lstUniqueNessVariants.Count.ToString());
            wr.Close();
        }
        void CalcAbsoluteSemsPerFile()
        {
            TextWriter wr = new StreamWriter("./out/AbsoluteSemsPerFile.txt");
            TextWriter writerHTML = new StreamWriter("./out/AbsoluteSemsPerFile.html");
            writerHTML.WriteLine("<html>\n<body>");

            foreach (string nameCurrentDomain in lstDomainNames)
            {
                wr.WriteLine("\n----------------" + nameCurrentDomain + "----------------");

                writerHTML.WriteLine("<h1>" + nameCurrentDomain + "</h1>");
                List<string> lstSemNamesInCurrentDomain = new List<string>();
                foreach (NessAttributes attr in listNesses)
                {
                    if (attr.domain == nameCurrentDomain)
                        if (!lstSemNamesInCurrentDomain.Contains(attr.sem)) lstSemNamesInCurrentDomain.Add(attr.sem); //polarity
                }
                lstSemNamesInCurrentDomain.Sort();

                wr.Write("*file\\sem*");
                writerHTML.WriteLine("<table border=1>\n<tr>");
                writerHTML.Write("<td>file\\sem</td>");
                foreach (string semName in lstSemNamesInCurrentDomain)
                {
                    wr.Write("\t" + semName);
                    writerHTML.Write("<td>" + semName + "</td>");
                }
                wr.WriteLine();
                writerHTML.WriteLine("\n</tr>");
                foreach (string nameCurrentFile in lstFileNames)
                {
                    wr.Write(nameCurrentFile);
                    writerHTML.Write("<tr>\n<td>" + nameCurrentFile + "</td>");
                    foreach (string nameCurrentSem in lstSemNamesInCurrentDomain)
                    {
                        int cntNessesInThisSemDomAndFile = 0;
                        foreach (NessAttributes attr in listNesses)
                        {
                            if ((attr.domain == nameCurrentDomain) && (attr.sem == nameCurrentSem) && (attr.filename == nameCurrentFile))
                                cntNessesInThisSemDomAndFile++;
                        }

                        wr.Write("\t" + cntNessesInThisSemDomAndFile.ToString());
                        writerHTML.Write("<td>" + cntNessesInThisSemDomAndFile.ToString() + "</td>");
                    }
                    wr.WriteLine();
                    writerHTML.WriteLine("\n</tr>");
                }
                writerHTML.WriteLine("</table>");

                //po strokam - fajly, po stlbtsam - semy.
            }
            wr.Close();
            writerHTML.WriteLine("</body>\n</html>");
            writerHTML.Close();
        }

        List<string> GetListOfSemsInCurrentDomain(string nameCurrentDomain)
        {
            List<string> lstSemNamesInCurrentDomain = new List<string>();
            foreach (NessAttributes attr in listNesses)
            {
                if (attr.domain == nameCurrentDomain)
                    if (!lstSemNamesInCurrentDomain.Contains(attr.sem)) lstSemNamesInCurrentDomain.Add(attr.sem); //polarity
            }
            return lstSemNamesInCurrentDomain;
        }
        void CalcKeynessPerDomain()
        {
            Dictionary<string, double> dicAvgSems = new Dictionary<string, double>();
            TextWriter wr = new StreamWriter("./out/KeynessNewPerDomainButStillDoubtful.txt");
            TextWriter writerHTML = new StreamWriter("./out/KeynessNewPerDomainButStillDoubtful.html");
            writerHTML.WriteLine("<html>\n<body>");
            writerHTML.WriteLine("<h1> Keyness </h1>");
            foreach (string nameCurrentDomain in lstDomainNames)
            {
                wr.WriteLine("\n----------------" + nameCurrentDomain + "----------------");
                writerHTML.WriteLine("<h2> " + nameCurrentDomain + " </h2>");
                // writerHTML.WriteLine("<h1>" + nameCurrentDomain + "</h1>");
                List<string> lstSemNamesInCurrentDomain = GetListOfSemsInCurrentDomain(nameCurrentDomain);

                lstSemNamesInCurrentDomain.Sort();
                wr.Write("**sem**");
                writerHTML.WriteLine("<table border=1>\n<tr>");
                writerHTML.Write("<td>file\\sem</td>");
                foreach (string semName in lstSemNamesInCurrentDomain)
                {
                    wr.Write("\t" + semName);
                    writerHTML.Write("<td>" + semName + "</td>");
                }
                wr.WriteLine();
                writerHTML.WriteLine("\n</tr>");
                foreach (string fileName in lstFileNames)
                {
                    wr.Write(fileName);
                    writerHTML.Write("<tr>\n<td>" + fileName + "</td>");
                    foreach (string semName in lstSemNamesInCurrentDomain)
                    {
                        int cntCurrentSemInThisFile = 0;
                        int cntCurrentSemInOtherPeriodsFiles = 0;
                        foreach (NessAttributes attr in listNesses)
                        {
                            //if ((fileName.ToLower() == "01_pamela") && (semName == "polite")) Console.WriteLine(".");
                            if ((attr.filename == fileName) && (attr.sem == semName))
                            {
                                cntCurrentSemInThisFile++; //polarity
                               // Console.WriteLine(".");
                            }
                            if ((attr.period != fileName.Substring(0, 2)) && (attr.sem == semName)) cntCurrentSemInOtherPeriodsFiles++; //polarity
                        }

                        int cntWordsOtherPeriods = 0;
                        double cntSemsAllInOtherFiles = 0;

                        foreach (string iFile in lstFileNames)
                        {
                            if (iFile[1] != fileName[1]) cntWordsOtherPeriods += dicWordsPerFile[iFile];
                            if (iFile[1] != fileName[1]) cntSemsAllInOtherFiles += dicSemsPerFile[iFile];
                        }
                        int cntWordsThis = dicWordsPerFile[fileName];
                        int cntWordsTotal = cntWordsThis + cntWordsOtherPeriods;
                        double valEstimatedThis = ((double)cntWordsThis * (double)(cntCurrentSemInOtherPeriodsFiles + cntCurrentSemInThisFile)) / (double)cntWordsTotal;
                        double valEstimatedOthers = ((double)cntWordsOtherPeriods * (double)(cntCurrentSemInThisFile + cntCurrentSemInOtherPeriodsFiles)) / (double)cntWordsTotal;
                        //wr.Write("\t" + cntSemThisFile.ToString());
                        //wr.Write("["+valEstimatedThis.ToString("F3")+"]");
                        double valDeviation = ((double)cntCurrentSemInThisFile - valEstimatedThis) * magicMultiplier / cntWordsThis;
                        double valSemPercents = (double)cntCurrentSemInThisFile * magicMultiplier / cntWordsThis;
                        double valSemPerSems = (double)cntCurrentSemInThisFile / dicSemsPerFile[fileName];
                        if (!dicAvgSems.ContainsKey(semName)) dicAvgSems.Add(semName, 0);
                        dicAvgSems[semName] += valSemPercents / lstFileNames.Count;
                        double cntSemsAllThisAndOthers = cntCurrentSemInOtherPeriodsFiles + dicSemsPerFile[fileName];
                        double a = cntCurrentSemInThisFile;
                        double b = cntCurrentSemInOtherPeriodsFiles;
                        //double c = dicSemsPerFile[fileName];// -cntCurrentSemInThisFile;
                        //double d = cntSemsAllInOtherFiles; // -cntCurrentSemInOtherPeriodsFiles;
                        double c = cntWordsThis;// -cntCurrentSemInThisFile;
                        double d = cntWordsOtherPeriods; // -cntCurrentSemInOtherPeriodsFiles;
                        double E1 = c * (a + b) / (c + d);
                        double E2 = d * (a + b) / (c + d);
                        double valLogLikelyHood = 2 * (((double)cntCurrentSemInThisFile * ((cntCurrentSemInThisFile != 0) ? Math.Log(a / E1) : 0)) + (b * ((b != 0) ? (Math.Log(b / E2)) : 0)));
                        double realtiveFreq1 = a / c;
                        double realtiveFreq2 = b / d;
                        if (realtiveFreq1 < realtiveFreq2) valLogLikelyHood = -valLogLikelyHood;

                        double valChiSquareStrange = (a * d - b * c) * (a * d - b * c) * (a + b + c + d) /
                            ((a + c) * (b + d) * (a + b) * (c + d));

                        double valChiNormal = ((double)cntCurrentSemInThisFile - valEstimatedThis) * (cntCurrentSemInThisFile - valEstimatedThis) / valEstimatedThis;
                        wr.Write("\t" + valLogLikelyHood.ToString("F3"));
                        writerHTML.Write("<td>" + valLogLikelyHood.ToString("F3") + "</td>");
                        //wr.Write("\t" + valSemPercents.ToString("F3"));
                        //wr.Write("\t" + cntWordsThis + ":" + cntSemThisFile.ToString() + ":" + valSemPercents.ToString("F3"));

                        //wr.Write("\t" + cntSemThisFile.ToString());
                        //wr.Write(valDeviation.ToString("F3"));
                        //debug
                        //Console.WriteLine(fileName);
                        if ((fileName.ToLower() == "02_pp") && (semName == "polite"))
                        {
                            Console.WriteLine("####Clarissa");
                            Console.WriteLine("wordsinThis=" + cntWordsThis.ToString());
                            //Console.WriteLine("attr.filename=" + );
                            Console.WriteLine("wordsinOther=" + cntWordsOtherPeriods.ToString());
                            Console.WriteLine("seminTHisTotal =" + dicSemsPerFile[fileName].ToString());
                            Console.WriteLine("seminTHis a=" + a.ToString());
                            Console.WriteLine("seminother b=" + b.ToString());
                            Console.WriteLine("othersemsinthisfile c=" + c.ToString());
                            Console.WriteLine("othersemsinotherfile d=" + d.ToString());
                            Console.WriteLine("seminOther=" + cntCurrentSemInOtherPeriodsFiles.ToString());
                            Console.WriteLine("a=" + a.ToString());
                            Console.WriteLine("b=" + b.ToString());
                            Console.WriteLine("c=" + c.ToString());
                            Console.WriteLine("d=" + d.ToString());
                            Console.WriteLine("E1=" + E1.ToString());
                            Console.WriteLine("E2=" + E2.ToString());


                            Console.WriteLine("loghood=" + valLogLikelyHood.ToString("F3"));
                        }
                    }
                    wr.WriteLine();
                    writerHTML.WriteLine("\n</tr>");
                }
                writerHTML.WriteLine("\n</table>");
            }
            //------------------------- calc stats per periods -------------------
            wr.WriteLine("\n################ averaged keyness and it's dispersion per period ###################");
            writerHTML.WriteLine("<h1>averaged keyness and it's dispersion per period</h1>");

            foreach (string nameCurrentDomain in lstDomainNames)
            {
                List<string> lstSemNamesInCurrentDomain = GetListOfSemsInCurrentDomain(nameCurrentDomain);
                wr.WriteLine("\n----------------" + nameCurrentDomain + "----------------");
                writerHTML.WriteLine("<h2>" + nameCurrentDomain + "</h2>"); 
                writerHTML.WriteLine("<table border=\"1\">\n<tr>\n<td>prd\\sem</td>"); 
                wr.Write("prd\\sem");
                foreach (string semName in lstSemNamesInCurrentDomain)
                {
                    wr.Write("\t" + semName + ":" + dicAvgSems[semName].ToString("F3"));
                    writerHTML.WriteLine("<td>" + semName + ":" + dicAvgSems[semName].ToString("F3") + "</td>"); 
                }
                wr.WriteLine();
                writerHTML.WriteLine("</tr>"); 
                foreach (string tokPeriod in strPeriods)
                {
                    wr.Write(tokPeriod);
                    writerHTML.WriteLine("<td>" + tokPeriod + "</td>"); 
                    int cntFileInCurrentPeriod = 0;
                    foreach (string tokFileName in lstFileNames)
                    {
                        if (tokFileName.StartsWith(tokPeriod))
                        {
                            cntFileInCurrentPeriod++;
                        }
                    }
                    foreach (string tokSem in lstSemNamesInCurrentDomain)
                    {
                        Dictionary<string, int> dicCurrentSemPerFile = new Dictionary<string, int>();
                        foreach (NessAttributes attr in listNesses)
                        {
                            //  Console.WriteLine(tokPeriod + ":" + tokSem+":"+attr.filename+":"+ attr.strSemPolarity);
                            if (attr.filename.StartsWith(tokPeriod) && (attr.sem == tokSem))//polarity
                            {

                                if (!dicCurrentSemPerFile.ContainsKey(attr.filename)) dicCurrentSemPerFile.Add(attr.filename, 0);
                                dicCurrentSemPerFile[attr.filename] += 1;
                            }
                            //calc freq per file

                            //calc average
                            //calc dispersion
                        }
                        double sumFreqCurrentPeriod = 0;
                        string strTempVals = "";
                        foreach (string tokFileName in dicCurrentSemPerFile.Keys)
                        {
                            sumFreqCurrentPeriod += dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName];
                            strTempVals += dicWordsPerFile[tokFileName] + ":" + dicCurrentSemPerFile[tokFileName].ToString() + ":" + (dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName]).ToString("F3") + "  ";
                            //sumFreqCurrentPeriod += dicCurrentSemPerFile[tokFileName];
                        }

                        double valAvgCurrentPeriod = sumFreqCurrentPeriod / cntFileInCurrentPeriod;
                        double valDispPeriod = 0;
                        foreach (string tokFileName in dicCurrentSemPerFile.Keys)
                        {
                            valDispPeriod += (dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName] - valAvgCurrentPeriod) * (dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName] - valAvgCurrentPeriod);
                        }
                        valDispPeriod = Math.Sqrt(valDispPeriod / cntFileInCurrentPeriod);

                        wr.Write("\t" + ((double)valAvgCurrentPeriod).ToString("F3") + "(" + ((double)valDispPeriod).ToString("F3") + ")");
                        writerHTML.WriteLine("<td>" + ((double)valAvgCurrentPeriod).ToString("F3") + "(" + ((double)valDispPeriod).ToString("F3") + ")" + "</td>"); 
                        //  wr.Write("\t" + strTempVals);
                    }
                    wr.WriteLine();
                    writerHTML.WriteLine("\n</tr>");
                    //wr.WriteLine("files in period="+cntFileInCurrentPeriod.ToString());
                }
                writerHTML.WriteLine("\n</table>");
            }
            writerHTML.Close();
            wr.Close();
        }

        void CalcKeyness()
        {
            Dictionary<string, double> dicAvgSems = new Dictionary<string, double>();
            TextWriter wr = new StreamWriter("./out/KeynessOldAndABitDoubtful.txt");
            wr.Write("**sem**");
            foreach (string semName in lstSemNames)
            {
                wr.Write("\t" + semName);
            }
            wr.WriteLine();
            foreach (string fileName in lstFileNames)
            {
                wr.Write(fileName);
                foreach (string semName in lstSemNames)
                {
                    int cntSemThisFile = 0;
                    int cntSemOtherFiles = 0;
                    foreach (NessAttributes attr in listNesses)
                    {
                        if ((attr.filename == fileName) && (attr.sem == semName)) cntSemThisFile++; //polarity
                        if ((attr.period != fileName.Substring(0, 2)) && (attr.sem == semName)) cntSemOtherFiles++; //polarity
                    }


                    int cntWordsOther = 0;
                    double cntSemsAllInOtherFiles = 0;

                    foreach (string iFile in lstFileNames)
                    {
                        if (iFile[1] != fileName[1]) cntWordsOther += dicWordsPerFile[iFile];
                        if (iFile[1] != fileName[1]) cntSemsAllInOtherFiles += dicSemsPerFile[iFile];
                    }
                    int cntWordsThis = dicWordsPerFile[fileName];
                    int cntWordsTotal = cntWordsThis + cntWordsOther;
                    double valEstimatedThis = ((double)cntWordsThis * (double)(cntSemOtherFiles + cntSemThisFile)) / (double)cntWordsTotal;
                    double valEstimatedOthers = ((double)cntWordsOther * (double)(cntSemThisFile + cntSemOtherFiles)) / (double)cntWordsTotal;
                    //wr.Write("\t" + cntSemThisFile.ToString());
                    //wr.Write("["+valEstimatedThis.ToString("F3")+"]");
                    double valLogLikelyHood = 2 * (((double)cntSemThisFile * ((cntSemThisFile != 0) ? Math.Log((double)cntSemThisFile / valEstimatedThis) : 0)) + ((double)cntSemOtherFiles * ((cntSemOtherFiles != 0) ? (Math.Log((double)cntSemOtherFiles / valEstimatedOthers)) : 0)));
                    double valDeviation = ((double)cntSemThisFile - valEstimatedThis) * magicMultiplier / cntWordsThis;
                    double valSemPercents = (double)cntSemThisFile * magicMultiplier / cntWordsThis;
                    double valSemPerSems = (double)cntSemThisFile / dicSemsPerFile[fileName];
                    if (!dicAvgSems.ContainsKey(semName)) dicAvgSems.Add(semName, 0);
                    dicAvgSems[semName] += valSemPercents / lstFileNames.Count;
                    double cntSemsAllThisAndOthers = cntSemOtherFiles + dicSemsPerFile[fileName];
                    double a = cntSemThisFile;
                    double b = cntSemOtherFiles;
                    double c = dicSemsPerFile[fileName] - cntSemThisFile;
                    double d = cntSemsAllInOtherFiles - cntSemOtherFiles;
                    double valChiSquareStrange = (a * d - b * c) * (a * d - b * c) * (a + b + c + d) /
                        ((a + c) * (b + d) * (a + b) * (c + d));

                    double valChiNormal = ((double)cntSemThisFile - valEstimatedThis) * (cntSemThisFile - valEstimatedThis) / valEstimatedThis;
                    //wr.Write("" + valLogLikelyHood.ToString("F3"));
                    wr.Write("\t" + valSemPercents.ToString("F3"));
                    //wr.Write("\t" + cntWordsThis + ":" + cntSemThisFile.ToString() + ":" + valSemPercents.ToString("F3"));

                    //wr.Write("\t" + cntSemThisFile.ToString());
                    //wr.Write(valDeviation.ToString("F3"));
                    //debug
                    if ((fileName == "01_Clarissa") && (semName == "moralp"))
                    {
                        Console.WriteLine("####Clarissa");
                        Console.WriteLine("wordsinThis=" + cntWordsThis.ToString());
                        Console.WriteLine("wordsinOther=" + cntWordsOther.ToString());
                        Console.WriteLine("seminTHisTotal =" + dicSemsPerFile[fileName].ToString());
                        Console.WriteLine("seminTHis a=" + a.ToString());
                        Console.WriteLine("seminother b=" + b.ToString());
                        Console.WriteLine("othersemsinthisfile c=" + c.ToString());
                        Console.WriteLine("othersemsinotherfile d=" + d.ToString());
                        Console.WriteLine("seminOther=" + cntSemOtherFiles.ToString());
                        Console.WriteLine("loghood=" + valLogLikelyHood.ToString("F3"));

                    }
                }
                wr.WriteLine();
            }
            //--- calc stats per periods ---
            wr.WriteLine("-------- dispersion per period");
            wr.Write("**sem**");
            foreach (string semName in lstSemNames)
            {
                wr.Write("\t" + semName + ":" + dicAvgSems[semName].ToString("F3"));
            }
            wr.WriteLine();
            foreach (string tokPeriod in strPeriods)
            {
                wr.Write(tokPeriod);
                int cntFileInCurrentPeriod = 0;
                foreach (string tokFileName in lstFileNames)
                {
                    if (tokFileName.StartsWith(tokPeriod))
                    {
                        cntFileInCurrentPeriod++;
                    }
                }
                foreach (string tokSem in lstSemNames)
                {
                    Dictionary<string, int> dicCurrentSemPerFile = new Dictionary<string, int>();
                    foreach (NessAttributes attr in listNesses)
                    {
                        //  Console.WriteLine(tokPeriod + ":" + tokSem+":"+attr.filename+":"+ attr.strSemPolarity);
                        if (attr.filename.StartsWith(tokPeriod) && (attr.sem == tokSem))//polarity
                        {

                            if (!dicCurrentSemPerFile.ContainsKey(attr.filename)) dicCurrentSemPerFile.Add(attr.filename, 0);
                            dicCurrentSemPerFile[attr.filename] += 1;
                        }
                        //calc freq per file

                        //calc average
                        //calc dispersion
                    }
                    double sumFreqCurrentPeriod = 0;
                    string strTempVals = "";
                    foreach (string tokFileName in dicCurrentSemPerFile.Keys)
                    {
                        sumFreqCurrentPeriod += dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName];
                        strTempVals += dicWordsPerFile[tokFileName] + ":" + dicCurrentSemPerFile[tokFileName].ToString() + ":" + (dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName]).ToString("F3") + "  ";
                        //sumFreqCurrentPeriod += dicCurrentSemPerFile[tokFileName];
                    }

                    double valAvgCurrentPeriod = sumFreqCurrentPeriod / cntFileInCurrentPeriod;
                    double valDispPeriod = 0;
                    foreach (string tokFileName in dicCurrentSemPerFile.Keys)
                    {
                        valDispPeriod += (dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName] - valAvgCurrentPeriod) * (dicCurrentSemPerFile[tokFileName] * magicMultiplier / dicWordsPerFile[tokFileName] - valAvgCurrentPeriod);
                    }
                    valDispPeriod = Math.Sqrt(valDispPeriod / cntFileInCurrentPeriod);

                    wr.Write("\t" + ((double)valAvgCurrentPeriod).ToString("F3") + ":" + ((double)valDispPeriod).ToString("F3"));
                    //  wr.Write("\t" + strTempVals);

                }

                wr.WriteLine();
            }
            
            wr.Close();

        }
        void WriteTblSemsPerFileToWriter(TextWriter wr, Dictionary<string, Dictionary<string, int>> table)
        {
            //get list of all files
            List<string> lstFiles = new List<string>();
            foreach (KeyValuePair<string, Dictionary<string, int>> dicSemInFiles in table)
            {
                foreach (KeyValuePair<string, int> element in dicSemInFiles.Value)
                {
                    if (!lstFiles.Contains(element.Key))
                        lstFiles.Add(element.Key);
                }
            }
            wr.Write("**Sem**");
            foreach (string fileName in lstFiles) wr.Write("\t" + fileName);
            wr.WriteLine();
            IEnumerable<KeyValuePair<string, Dictionary<string, int>>> queue;
            queue = table.OrderBy(pair => pair.Key);
            foreach (KeyValuePair<string, Dictionary<string, int>> dicSemInFiles in queue)
            {
                wr.Write(dicSemInFiles.Key);

                foreach (string fileName in lstFiles)
                {
                    int cntNess = 0;
                    if (dicSemInFiles.Value.ContainsKey(fileName))
                    {
                        cntNess = dicSemInFiles.Value[fileName];
                    }
                    wr.Write("\t" + cntNess.ToString());
                }
                wr.WriteLine();
            }
        }

        void WriteTblKeynessToWriter(TextWriter wr, Dictionary<string, Dictionary<string, double>> table)
        {
            //get list of all files
            List<string> lstSems = new List<string>();
            foreach (KeyValuePair<string, Dictionary<string, double>> dicSemInFiles in table)
            {
                foreach (KeyValuePair<string, double> element in dicSemInFiles.Value)
                {
                    if (!lstSems.Contains(element.Key))
                        lstSems.Add(element.Key);
                }
            }
            wr.Write("**Sem**");
            foreach (string semName in lstSems) wr.Write("\t" + semName);
            wr.WriteLine();
            IEnumerable<KeyValuePair<string, Dictionary<string, double>>> queue;
            queue = table.OrderBy(pair => pair.Key);
            foreach (KeyValuePair<string, Dictionary<string, double>> dicSemInFile in queue)
            {
                wr.Write(dicSemInFile.Key);
                double total = 0;
                foreach (string semName in lstSems)
                {
                    if (dicSemInFile.Value.ContainsKey(semName))
                    {
                        wr.Write("\t" + dicSemInFile.Value[semName].ToString("F3"));
                        total += dicSemInFile.Value[semName];
                    }
                    else
                        wr.Write("\t 0.00");

                }
                wr.WriteLine("  ### " + total.ToString());
                //                wr.WriteLine();
            }
        }

        void WriteStatsToFile(string fileName, Dictionary<string, Dictionary<string, int>> table, int Max)
        {
            TextWriter wr = new StreamWriter(fileName);
            WriteStatsToWriter(wr, table, Max);
            wr.Close();
        }

        void WriteTblSemsPerFileToFile(string fileName, Dictionary<string, Dictionary<string, int>> table)
        {
            TextWriter wr = new StreamWriter(fileName);
            WriteTblSemsPerFileToWriter(wr, table);
            wr.Close();
        }
        void WriteTblKeynessToFile(string fileName, Dictionary<string, Dictionary<string, double>> table)
        {
            TextWriter wr = new StreamWriter(fileName);
            WriteTblKeynessToWriter(wr, table);
            wr.Close();
        }
        void WriteStatsToWriter(TextWriter wr, Dictionary<string, Dictionary<string, int>> table, int Max)
        {
            foreach (KeyValuePair<string, Dictionary<string, int>> dic_dom in table)
            {
                wr.WriteLine("\nDomain: " + dic_dom.Key + "     -------------------------------");
                IEnumerable<KeyValuePair<string, int>> queue;
                queue = dic_dom.Value.OrderByDescending(pair => pair.Value);
                foreach (KeyValuePair<string, int> element in queue)
                {
                    if (element.Key.StartsWith("sem."))
                        wr.WriteLine(element.Key + ":\t" + ((double)element.Value * 100 / Max).ToString("F2"));
                }
                foreach (KeyValuePair<string, int> element in queue)
                {
                    if (!element.Key.StartsWith("sem."))
                        wr.WriteLine(element.Key + "\t" + element.Value.ToString() + "\t" + ((double)element.Value * 100 / Max).ToString("F2"));
                }
                wr.WriteLine();
            }
        }

        void WriteFreqsToWriter(TextWriter wr, Dictionary<string, int> table, int Max)
        {
            IEnumerable<KeyValuePair<string, int>> queue;
            queue = table.OrderBy(pair => pair.Key);
            foreach (KeyValuePair<string, int> element in queue)
            {
                wr.WriteLine(element.Key + "\t" + element.Value.ToString());
            }
            wr.WriteLine();
        }

        void WriteAttrCToFile(TextWriter wr, Dictionary<string, Dictionary<string, int>> table, int cntNess)
        {
            foreach (KeyValuePair<string, Dictionary<string, int>> dic_dom in table)
            {

                IEnumerable<KeyValuePair<string, int>> queue;
                queue = dic_dom.Value.OrderByDescending(pair => pair.Value);
                //calc ttrC count
                int cntAttrC = 0;
                foreach (KeyValuePair<string, int> element in queue)
                {
                    cntAttrC += element.Value;
                }
                wr.WriteLine("\nSem: " + dic_dom.Key + " " + cntAttrC.ToString());
                foreach (KeyValuePair<string, int> element in queue)
                {
                    wr.WriteLine(element.Key + ":\t" + ((double)element.Value * 100 / cntAttrC).ToString("F2"));
                }
                //  wr.WriteLine();

            }
        }


        void WriteCPerPeriodToFile(string filename, Dictionary<string, Dictionary<string, Dictionary<string, int>>> dataset)
        {
            TextWriter wr = new StreamWriter(filename);
            //IEnumerable<KeyValuePair<string, int>> queue;
            //queue = dic_dom.Value.OrderByDescending(pair => pair.Value);

            string[] nessesOfInterest = new string[] { "goodness", "loneliness", "darkness", "consciousness", "business", "happiness", "kindness" };

            foreach (KeyValuePair<string, Dictionary<string, Dictionary<string, int>>> dicPeriodsPerSem in dataset)
            {
                if (nessesOfInterest.Contains(dicPeriodsPerSem.Key))
                {
                    wr.WriteLine("\n" + dicPeriodsPerSem.Key);
                    wr.WriteLine("\t1720-1780\t1780-1830\t1830-1880\t1880-1915\t1915-1945\t1945-");
                    int cntTotal = 0;
                    foreach (KeyValuePair<string, Dictionary<string, int>> attrsPerPeriod in dicPeriodsPerSem.Value)
                    {
                        //  string[] strPeriods = new string[] { "01", "02", "03", "04", "05", "06" };
                        foreach (string strPeriod in strPeriods)
                        {
                            if (attrsPerPeriod.Value.ContainsKey(strPeriod))
                                cntTotal += attrsPerPeriod.Value[strPeriod];
                        }
                    }
                    foreach (KeyValuePair<string, Dictionary<string, int>> attrsPerPeriod in dicPeriodsPerSem.Value)
                    {
                        wr.Write(attrsPerPeriod.Key + "\t");

                        foreach (string strPeriod in strPeriods)
                        {
                            if (attrsPerPeriod.Value.ContainsKey(strPeriod))
                                wr.Write(((float)attrsPerPeriod.Value[strPeriod] / cntTotal * 100).ToString("F2") + "\t");
                            else
                                wr.Write("0\t");
                        }
                        wr.WriteLine();
                    }
                    //  wr.WriteLine();
                }
            }
            wr.Close();
        }
        void WriteAveregedSemToFile(TextWriter wr, Dictionary<string, List<double>> table, int cntFile)
        {
            Dictionary<string, double> averegedValues = new Dictionary<string, double>();
            foreach (KeyValuePair<string, List<double>> element in table)
            {
                double averege = 0;
                foreach (double val in element.Value)
                {
                    averege += val;
                }
                averegedValues.Add(element.Key, averege);
            }
            IEnumerable<KeyValuePair<string, double>> queue;
            queue = averegedValues.OrderByDescending(pair => pair.Value);
            //calc ttrC count
            foreach (KeyValuePair<string, double> element in queue)
            {

                wr.WriteLine(element.Key + ":\t" + (element.Value * 100).ToString("F2"));
            }
            wr.WriteLine();
        }
        void WriteNessGroupsToFile(string filename, Dictionary<string, Dictionary<string, int>> table)
        {
            TextWriter wr = new StreamWriter(filename);
            foreach (KeyValuePair<string, Dictionary<string, int>> dic_dom in table)
            {
                wr.Write(dic_dom.Key + "\t");
                IEnumerable<KeyValuePair<string, int>> queue;
                queue = dic_dom.Value.OrderByDescending(pair => pair.Value);
                foreach (KeyValuePair<string, int> element in queue)
                {

                    wr.Write(element.Key + "[" + element.Value.ToString() + "] ");
                }
                wr.WriteLine();
            }
            wr.Close();
        }

        void WriteNessPerPeriodToFile(string filename, Dictionary<string, Dictionary<string, int>> table)
        {
            TextWriter wr = new StreamWriter(filename);
            foreach (KeyValuePair<string, Dictionary<string, int>> dic_sems in table)
            {
                wr.Write(dic_sems.Key + "\t");
                //           IEnumerable<KeyValuePair<string, int>> queue;
                //            queue = dic_dom.Value.OrderBy(pair => pair.Key);
                //             foreach (KeyValuePair<string, int> element in queue)
                //              {
                //                    wr.Write(element.Key + ":" + element.Value.ToString() + ",");
                //              }
                foreach (string strPeriod in strPeriods)
                {
                    if (dic_sems.Value.ContainsKey(strPeriod))
                        wr.Write(strPeriod + ":" + dic_sems.Value[strPeriod].ToString() + ",");
                    else
                        wr.Write(strPeriod + ":0,");
                }

                wr.WriteLine();
            }
            wr.Close();
        }
        void WriteNessGroupsPeriodToFile(string filename, Dictionary<string, Dictionary<string, int>> table, Dictionary<string, Statistics> dicStatsPerPeriod)
        {
            TextWriter wr = new StreamWriter(filename);
            foreach (KeyValuePair<string, Dictionary<string, int>> dic_sem in table)
            {
                wr.Write(dic_sem.Key + "\t");
                IEnumerable<KeyValuePair<string, int>> queue;
                queue = dic_sem.Value.OrderByDescending(pair => pair.Value);
                foreach (KeyValuePair<string, int> element in queue)
                {

                    wr.Write(element.Key + "[" + element.Value.ToString() + " (");
                    foreach (string period in strPeriods)
                    {
                        int cntThisNessInPeriod = 0;
                        try
                        {
                            cntThisNessInPeriod = dicStatsPerPeriod[period].tblNessGroups[dic_sem.Key][element.Key];
                        }
                        catch { }
                        wr.Write(period + ":" + cntThisNessInPeriod + " ");
                    }
                    wr.Write(")]  ");
                }
                wr.WriteLine();
            }
            wr.Close();
        }

        void WriteNessNominativenessToFile(string filename, Dictionary<string, Dictionary<string, int>> table)
        {//tokens and nesses per sem
            TextWriter wr = new StreamWriter(filename);
            foreach (KeyValuePair<string, Dictionary<string, int>> dic_dom in table)
            {
                wr.Write(dic_dom.Key + "\t");
                IEnumerable<KeyValuePair<string, int>> queue;
                queue = dic_dom.Value.OrderByDescending(pair => pair.Value);
                int cntNesses = dic_dom.Value.Count;
                wr.Write(cntNesses + "\t");
                int cntWords = 0;
                foreach (KeyValuePair<string, int> element in queue)
                {
                    cntWords += element.Value;
                    //wr.Write(element.Key + "[" + element.Value.ToString() + "] ");
                }
                wr.WriteLine(cntWords.ToString());
            }
            wr.Close();
        }


        void WriteHTMLTablesToFile(string filename, Dictionary<string, Dictionary<string, int>>[] tables, int maxVal)
        {
            TextWriter wr = new StreamWriter(filename);
            wr.WriteLine("<html>");
            foreach (Dictionary<string, Dictionary<string, int>> table in tables)
            {
                WriteHtmlTableToWriter(wr, table, maxVal);
                wr.WriteLine("<br /> <br />\n");
            }

            wr.WriteLine("</html>");
            wr.Close();

        }

        void WriteHtmlTableToWriter(TextWriter wr, Dictionary<string, Dictionary<string, int>> table, int maxVal)
        {

            //create column names
            List<string> colNames = new List<string>();
            foreach (KeyValuePair<string, Dictionary<string, int>> row in table)
                foreach (KeyValuePair<string, int> colName in row.Value)
                {
                    if (!colNames.Contains(colName.Key)) colNames.Add(colName.Key);
                }
            colNames.Sort();
            wr.WriteLine("<table  border=\"1\" style=\"border: 1px solid black;  border-collapse: collapse;\">");
            wr.Write("<tr>\n<td>&nbsp;</td>");
            foreach (string colName in colNames) //writig header
                wr.Write("<th>" + colName + "</th>");
            //print rows
            IEnumerable<KeyValuePair<string, Dictionary<string, int>>> queue;
            queue = table.OrderByDescending(pair => pair.Key);
            foreach (KeyValuePair<string, Dictionary<string, int>> row in queue)
            {
                wr.Write("<tr>\n");
                wr.Write("\t<td>" + row.Key + "</td>");
                foreach (string colName in colNames) //writig vals
                {
                    if (row.Value.ContainsKey(colName))
                    {
                        double val = ((double)row.Value[colName] * 100) / maxVal;
                        // cheats!!!!!!!!!!!!!!!!!!!1
                        if (colName == "rcmod") val = val * 0.85;
                        if (colName == "mark") val = val * 0.92;
                        if (colName == "nsubjpass") val = val * 0.92;
                        if (colName == "partmod") val = val * 0.60;

                        if (val > 5)
                            wr.Write("<td style=\"color:red;\">" + val.ToString("F2") + "</td>");
                        else
                        {
                            if (val > 1)
                                wr.Write("<td style=\"color:#009000;\">" + val.ToString("F2") + "</td>");

                            else
                                wr.Write("<td>" + val.ToString("F2") + "</td>");
                        }
                    }
                    else
                        wr.Write("<td>0</td>");
                }
                wr.Write("\n</tr>\n");
            }

            wr.WriteLine("\n</tr>");
            wr.WriteLine("</table>");

        }
        void WriteHtmlSemPerFileToWriter(TextWriter wr, Dictionary<string, Dictionary<string, int>> table)
        {

            //create column names
            List<string> colNames = new List<string>();
            foreach (KeyValuePair<string, Dictionary<string, int>> row in table)
                foreach (KeyValuePair<string, int> colName in row.Value)
                {
                    if (!colNames.Contains(colName.Key)) colNames.Add(colName.Key);
                }
            colNames.Sort();
            wr.WriteLine("<table  border=\"1\" style=\"border: 1px solid black;  border-collapse: collapse;\">");
            wr.Write("<tr>\n<td>&nbsp;</td>");
            foreach (string colName in colNames) //writig header
                wr.Write("<th>" + colName + "</th>");
            //print rows
            IEnumerable<KeyValuePair<string, Dictionary<string, int>>> queue;
            queue = table.OrderBy(pair => pair.Key);
            foreach (KeyValuePair<string, Dictionary<string, int>> row in queue)
            {
                wr.Write("<tr>\n");
                wr.Write("\t<td>" + row.Key + "</td>");
                foreach (string colName in colNames) //writig vals
                {
                    if (row.Value.ContainsKey(colName))
                    {
                        double val = ((double)row.Value[colName] * 100) / dicNessPerFile[colName];


                        if (val > 5)
                            wr.Write("<td style=\"color:red;\">" + val.ToString("F2") + "</td>");
                        else
                        {
                            if (val > 1)
                                wr.Write("<td style=\"color:#009000;\">" + val.ToString("F2") + "</td>");

                            else
                                wr.Write("<td>" + val.ToString("F2") + "</td>");
                        }
                    }
                    else
                        wr.Write("<td>0</td>");
                }
                wr.Write("\n</tr>\n");
            }

            wr.WriteLine("\n</tr>");
            wr.WriteLine("</table>");

        }


        public void Process()
        {
            Console.WriteLine("GateExtractor 0.2");
            DateTime dt_start = DateTime.Now;

            //global init

            statsGlobal.tblNessPerSemPerDomain.Add("#GLOBAL#", new Dictionary<string, int>());
            statsGlobal.tblNessPerSemPerDomainWithoutPolarity.Add("#GLOBAL#", new Dictionary<string, int>());
            DirectoryInfo di = new DirectoryInfo("./in");
            FileInfo[] Files = di.GetFiles();
            try
            {
                if (Directory.Exists("./out")) Directory.Delete("./out", true);
            }
            catch (Exception ex)
            {
                Console.WriteLine("./out :" + ex.Message);
            }
            try { Directory.CreateDirectory("./out"); }
            catch (Exception ex) { Console.WriteLine("./out :" + ex.Message); };

            foreach (FileInfo fi in Files)
            {
                ProcessFile(fi);
                cntFiles++;
            }
            //global results
            // print all groupds
            TextWriter wr = new StreamWriter("./out/___global");
            WriteStatsToWriter(wr, statsGlobal.tblNessPerSemPerDomain, numNessesGlobal);
            wr.WriteLine("############### SEMS WITHOUT POLARITY #################");
            WriteStatsToWriter(wr, statsGlobal.tblNessPerSemPerDomainWithoutPolarity, numNessesGlobal);
            wr.WriteLine("############### Attributes of interest #################");
            foreach (KeyValuePair<string, Dictionary<string, Dictionary<string, int>>> kvp in statsGlobal.tblAttrsOfInterest)
            {
                wr.WriteLine("\n--------------- " + kvp.Key + "-----------------");
                WriteAttrCToFile(wr, kvp.Value, numNessesGlobal);
            }
            wr.WriteLine("\n############### averaged sems percents #################\n");
            WriteAveregedSemToFile(wr, statsGlobal.tblSemPercentsPerFileOLD, cntFiles);
            //write averege values.
            wr.Close();
            WriteTblSemsPerFileToFile("./out/___globalSemsPerFile.txt", statsGlobal.tblSemsPerFile);
            //recalc keyness here
            Dictionary<string, Dictionary<string, double>> tblAveragedSemsPerPeriod = new Dictionary<string, Dictionary<string, double>>();

            foreach (KeyValuePair<string, Statistics> statsPeriod in dicStatsPerPeriod)
            {

                Dictionary<string, double> averagedSemsInThisPeriod = new Dictionary<string, double>();
                foreach (KeyValuePair<string, Dictionary<string, double>> dicSemsInFile in statsPeriod.Value.tblSemsPercentsPerFile)
                {
                    int numFilesInThisPeriod = statsPeriod.Key.Count();
                    foreach (KeyValuePair<string, double> sems in dicSemsInFile.Value)
                    {
                        if (!averagedSemsInThisPeriod.ContainsKey(sems.Key))
                            averagedSemsInThisPeriod.Add(sems.Key, 0);
                        averagedSemsInThisPeriod[sems.Key] += sems.Value / numFilesInThisPeriod;
                    }
                }
                tblAveragedSemsPerPeriod.Add(statsPeriod.Key, averagedSemsInThisPeriod);
            }
            WriteTblKeynessToFile("./out/___!!KEYNESS.txt", tblAveragedSemsPerPeriod);
            WriteCPerPeriodToFile("./out/___global_CPeriods.txt", statsGlobal.tblAttrPereriodPerPerNess);

            //    WriteHTMLTablesToFile("./out/___global.html", new Dictionary<string, Dictionary<string, int>>[] { tblDepsPerSemGlobal, tblDepsPerSemGlobalWOPol, tblDepsPerDomGlobal }, numNessesGlobal);
            dicNessPerFile.Add("total", numNessesGlobal);
            //  wr=new StreamWriter("./out/___global_SemsPerFile.html");
            //WriteHtmlSemPerFileToWriter(wr,tblSemsPerFile);
            ///wr.Close(); 
            WriteNessGroupsPeriodToFile("./out/___global.nesses", statsGlobal.tblNessGroups, dicStatsPerPeriod);
            WriteNessPerPeriodToFile("./out/___global.periods", statsGlobal.tblSemsPerPeriod);
            WriteNessNominativenessToFile("./out/___global.nominat", statsGlobal.tblNessGroups);
            //  WriteNessGroupsToFile("./out/___global.prefixes", tblNessPrefixesGlobal);
            //  WriteNessGroupsToFile("./out/___global.postfixes", tblNessPostfixesGlobal);
            wr = new StreamWriter("./out/___global_Freqs.txt");
            WriteFreqsToWriter(wr, statsGlobal.dicSemFreq, 1);
            wr.Close();

            DoInitGlobalStats();
            //CalcKeyness();
            CalcKeynessPerDomain();
            CalcAbsoluteSemsPerFile();
            CalcGlobalStatsNewFormat();
        }


        void ProcessFile(FileInfo fi)
        {
            Console.WriteLine("\nProcessing " + fi.Name);
            string fileName = fi.Name.Replace(".xml", "");
            //int numWords = 0;
            TextWriter wr = new StreamWriter("./out/" + fileName + ".sems");
            try
            {
                string strBufKeynessSem = "";
                string strBufKeynessNess = "";

                Dictionary<string, Dictionary<string, int>> nessGropusLocal = new Dictionary<string, Dictionary<string, int>>();

                XmlDocument myDoc = new XmlDocument();
                //init
                Statistics statsLocal = new Statistics();
                //tblDomainsLocal = new Dictionary<string, Dictionary<string, int>>();
                //Dictionary<string, Dictionary<string, int>> tblDepsPerSemLocal = new Dictionary<string, Dictionary<string, int>>();
                //Dictionary<string, Dictionary<string, int>> tblDepsPerDomLocal = new Dictionary<string, Dictionary<string, int>>();
                //Dictionary<string, Dictionary<string, int>> tblDepsPerSemLocalWOPol = new Dictionary<string, Dictionary<string, int>>();
                //Dictionary<string, Dictionary<string, int>> tblNessPrefixesLocal = new Dictionary<string, Dictionary<string, int>>();
                //Dictionary<string, Dictionary<string, int>> tblNessPostfixesLocal = new Dictionary<string, Dictionary<string, int>>();
                Dictionary<string, int> tblSemInCurretnFile = new Dictionary<string, int>();

                statsLocal.tblNessPerSemPerDomain.Add("#GLOBAL#", new Dictionary<string, int>());
                statsLocal.tblNessPerSemPerDomainWithoutPolarity.Add("#GLOBAL#", new Dictionary<string, int>());
                // nodesSyntaxTreeNodes = myDoc.SelectNodes("//Annotation[@Type='SyntaxTreeNode']");
                //load text
                myDoc.PreserveWhitespace = true;
                myDoc.Load(fi.FullName);
                Console.WriteLine("xml loaded");


                //load features
                XmlNodeList nodesNess = myDoc.SelectNodes("//ness");
                int cntNessNodes = nodesNess.Count;
                int numNessesLocal = cntNessNodes;
                numNessesGlobal += cntNessNodes;
                dicNessPerFile.Add(fileName, numNessesLocal);

                //wr.Write(text.InnerText);
                string period = fileName.Substring(0, 2);
                Console.WriteLine(nodesNess.Count.ToString() + " ness annotations selected");
                if (!dicStatsPerPeriod.ContainsKey(period))
                    dicStatsPerPeriod.Add(period, new Statistics());
                Statistics statsThisPeriod = dicStatsPerPeriod[period];

                for (int i = 0; i < nodesNess.Count; i++)
                {

                    XmlNode nodeCurrentNess = nodesNess[i];

                    string[] skipWords = new string[] { "of", "and", "or", "is", "was", "were", "has", "will", "shall", "in", "about" };

                    // wr.WriteLine("exploring ness \"" + strNessText+"\"");
                    NessAttributes currentNessAttributes = new NessAttributes();
                    //print ness props
                    currentNessAttributes.sem = "__ERROR";
                    currentNessAttributes.str = nodeCurrentNess.InnerText.Trim().ToLower();
                    currentNessAttributes.filename = fileName;
                    currentNessAttributes.period = period;
                    string polarity = "__ERROR";
                    currentNessAttributes.domain = "__ERROR";

                    try
                    {
                        currentNessAttributes.sem = nodeCurrentNess.Attributes["sem"].Value.Trim().ToLower();
                        polarity = nodeCurrentNess.Attributes["p"].Value;
                        currentNessAttributes.domain = nodeCurrentNess.Attributes["dom"].Value;
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("Error fetching attribute for ness " + currentNessAttributes.str + " - ness #" + i.ToString());
                    }
                    string[] attrsOfInterestList = new string[] { "ccomp", "cop", "nn", "infmod", "det", "predet", "poss", "dobj", "pobj", "prep", "nsubj", "amod", "partmod", "rcmod", "advmod", "conj", "xwf", "c" };
                    Dictionary<string, string> attrsOfInterestFetched = new Dictionary<string, string>();
                    foreach (string attrOfInterest in attrsOfInterestList)
                    {
                        string attrCurr = "";
                        try
                        {
                            attrCurr = nodeCurrentNess.Attributes[attrOfInterest].Value;
                        }
                        catch (Exception ex)
                        { }
                        if (attrCurr != "")
                        {
                            attrsOfInterestFetched.Add(attrOfInterest, attrCurr);
                        }
                    }
                    if (polarity == "+") polarity = "p";
                    if (polarity == "-") polarity = "m";
                    currentNessAttributes.strSemPolarity = currentNessAttributes.sem + polarity;
                    string semConcatenated = currentNessAttributes.strSemPolarity;
                    strBufKeynessSem += currentNessAttributes.strSemPolarity + "\n";
                    strBufKeynessNess += currentNessAttributes.str + "\n";
                    //accumulate all attr of interest
                    foreach (KeyValuePair<string, string> kvpAtrs in attrsOfInterestFetched)
                    {
                        if (!statsGlobal.tblAttrsOfInterest.ContainsKey(kvpAtrs.Key))
                            statsGlobal.tblAttrsOfInterest.Add(kvpAtrs.Key, new Dictionary<string, Dictionary<string, int>>());
                        if (!statsLocal.tblAttrsOfInterest.ContainsKey(kvpAtrs.Key))
                            statsLocal.tblAttrsOfInterest.Add(kvpAtrs.Key, new Dictionary<string, Dictionary<string, int>>());

                        AccumulateTableIncrement(statsLocal.tblAttrsOfInterest[kvpAtrs.Key], currentNessAttributes.strSemPolarity, kvpAtrs.Value);
                        AccumulateTableIncrement(statsGlobal.tblAttrsOfInterest[kvpAtrs.Key], currentNessAttributes.strSemPolarity, kvpAtrs.Value);

                    }
                    if (attrsOfInterestFetched.ContainsKey("c"))
                        AccumulateTableOfTablesIncrement(statsGlobal.tblAttrPereriodPerPerNess, currentNessAttributes.str, attrsOfInterestFetched["c"], period);
                    AccumulateTableIncrement(statsLocal.tblNessPerSemPerDomain, currentNessAttributes.domain, semConcatenated);
                    AccumulateTableIncrement(statsLocal.tblNessPerSemPerDomain, "#GLOBAL#", semConcatenated);
                    AccumulateTableIncrement(statsLocal.tblNessPerSemPerDomainWithoutPolarity, currentNessAttributes.domain, currentNessAttributes.sem);
                    AccumulateTableIncrement(statsLocal.tblNessPerSemPerDomainWithoutPolarity, "#GLOBAL#", currentNessAttributes.sem);

                    AccumulateTableIncrement(statsGlobal.tblSemsPerFile, currentNessAttributes.strSemPolarity, fileName);

                    AccumulateTableIncrement(statsGlobal.tblNessPerSemPerDomain, currentNessAttributes.domain, semConcatenated);
                    AccumulateTableIncrement(statsGlobal.tblNessPerSemPerDomain, "#GLOBAL#", semConcatenated);
                    AccumulateTableIncrement(statsGlobal.tblNessPerSemPerDomainWithoutPolarity, currentNessAttributes.domain, currentNessAttributes.sem);
                    AccumulateTableIncrement(statsGlobal.tblNessPerSemPerDomainWithoutPolarity, "#GLOBAL#", currentNessAttributes.sem);
                    AccumulateSimpleIncrement(tblSemInCurretnFile, semConcatenated);
                    AccumulateSimpleIncrement(statsGlobal.dicSemFreq, currentNessAttributes.strSemPolarity);

                    AccumulateTableIncrement(statsGlobal.tblSemsPerPeriod, currentNessAttributes.strSemPolarity, period);

                    listNesses.Add(currentNessAttributes);
                    //      AccumulateTableIncrement(tblNessPrefixesLocal, semConcatenated, strNessPrefix);
                    //      AccumulateTableIncrement(tblNessPrefixesGlobal, semConcatenated, strNessPrefix);
                    //       AccumulateTableIncrement(tblNessPostfixesLocal, semConcatenated, strNessPostfix);
                    ///    AccumulateTableIncrement(tblNessPostfixesGlobal, semConcatenated, strNessPostfix);

                    //   AccumulateTableIncrement(tblSemsPerFile, sem + polarity, fileName);
                    //  AccumulateTableIncrement(tblSemsPerFile, sem + polarity, "total");


                    List<string> lstProcessedFeatures = new List<string>();
                    //count deps
                    /*
                    foreach (XmlNode nessFeature in nodeCurrentNess.SelectNodes("Feature"))
                    {
                        featureName = nessFeature.SelectSingleNode("Name").InnerText;
                        featureVal = nessFeature.SelectSingleNode("Value").InnerText.ToLower();
                        string[] DepsOfInterestList = new string[] { "xTheme", "xWF", "amod", "ccomp", "conj", "det", "det[a]", "det[the]", "dobj", "expl", "infmod", "mark", "neg", "nsubj", "nsubjpass", "num", "partmod", "pobj", "poss", "predet", "prep", "rcmod", "rel" };
 
                        if (!DepsOfInterestList.Contains(featureName)) continue;
                         
                        if (featureName[0] != '_')
                            if (featureName != "string")
                            {
                                
                                AccumulateTableIncrement(tblDomainsLocal, dom, featureName + "." + featureVal);
                                AccumulateTableIncrement(tblDomainsLocal, "#GLOBAL#", featureName + "." + featureVal);
                                AccumulateTableIncrement(tblDomainsGlobal, dom, featureName + "." + featureVal);
                                AccumulateTableIncrement(tblDomainsGlobal, "#GLOBAL#", featureName + "." + featureVal);

                                AccumulateTableIncrement(tblDepsPerSemGlobal, semConcatenated, featureName);
                                AccumulateTableIncrement(tblDepsPerSemGlobal, semConcatenated + polarity, featureName);
                                AccumulateTableIncrement(tblDepsPerSemGlobal, "dom." + dom, featureName);
                                AccumulateTableIncrement(tblDepsPerDomGlobal, "dom." + dom, featureName);
                                AccumulateTableIncrement(tblDepsPerDomGlobal, "__glb", featureName);
                                AccumulateTableIncrement(tblDepsPerSemGlobalWOPol, semConcatenated, featureName);

                                AccumulateTableIncrement(tblNessGroupsGlobal, featureName, strNessText);

                                AccumulateTableIncrement(tblDepsPerSemLocal, semConcatenated, featureName);
                                AccumulateTableIncrement(tblDepsPerSemLocal, semConcatenated + polarity, featureName);
                                AccumulateTableIncrement(tblDepsPerSemLocal, "dom." + dom, featureName);
                                AccumulateTableIncrement(tblDepsPerDomLocal, "dom." + dom, featureName);
                                AccumulateTableIncrement(tblDepsPerDomLocal, "__glb", featureName);
                                AccumulateTableIncrement(tblDepsPerSemLocalWOPol, semConcatenated, featureName);
                            }
                    }

                    */

                    if (!nessGropusLocal.ContainsKey(currentNessAttributes.sem)) nessGropusLocal.Add(currentNessAttributes.sem, new Dictionary<string, int>());
                    Dictionary<string, int> groupNesses = nessGropusLocal[currentNessAttributes.sem];
                    AccumulateSimpleIncrement(groupNesses, currentNessAttributes.str);

                    AccumulateTableIncrement(statsGlobal.tblNessGroups, currentNessAttributes.strSemPolarity, currentNessAttributes.str);
                    AccumulateTableIncrement(statsLocal.tblNessGroups, currentNessAttributes.strSemPolarity, currentNessAttributes.str);
                    AccumulateTableIncrement(statsThisPeriod.tblNessGroups, currentNessAttributes.strSemPolarity, currentNessAttributes.str);


                }//end processing nesses
                //printing personage info
                WriteStatsToWriter(wr, statsLocal.tblNessPerSemPerDomain, numNessesLocal);
                wr.WriteLine("############### Attributes of interest #################");
                foreach (KeyValuePair<string, Dictionary<string, Dictionary<string, int>>> kvp in statsLocal.tblAttrsOfInterest)
                {
                    wr.WriteLine("--------------- " + kvp.Key + "-----------------");
                    WriteAttrCToFile(wr, kvp.Value, numNessesGlobal);
                }
                WriteStatsToFile("./out/" + fileName + ".wopol", statsLocal.tblNessPerSemPerDomainWithoutPolarity, numNessesLocal);

                //WriteHTMLTablesToFile("./out/" + fi.Name + ".html", new Dictionary<string, Dictionary<string, int>>[] { tblDepsPerSemLocal, tblDepsPerSemLocalWOPol, tblDepsPerDomLocal }, cntNessNodes);
                // WriteNessGroupsToFile("./out/" + fi.Name + ".prefix", tblNessPrefixesLocal);
                // WriteNessGroupsToFile("./out/" + fi.Name + ".postfix", tblNessPostfixesLocal);
                WriteNessGroupsToFile("./out/" + fileName + ".nesses", statsLocal.tblNessGroups);
                WriteNessNominativenessToFile("./out/" + fileName + ".nominat", statsLocal.tblNessGroups);
                TextWriter wrKeyness = new StreamWriter("./out/" + fileName + ".keySem");
                wrKeyness.WriteLine(strBufKeynessSem);
                wrKeyness.Close();
                wrKeyness = new StreamWriter("./out/" + fileName + ".keyNess");
                wrKeyness.WriteLine(strBufKeynessNess);
                wrKeyness.Close();
                //fill percents per file 
                foreach (KeyValuePair<string, int> element in tblSemInCurretnFile)
                {
                    double percentsSemCurrentFile = (double)element.Value / cntNessNodes;
                    if (!statsGlobal.tblSemPercentsPerFileOLD.ContainsKey(element.Key))
                        statsGlobal.tblSemPercentsPerFileOLD.Add(element.Key, new List<double>());
                    statsGlobal.tblSemPercentsPerFileOLD[element.Key].Add(percentsSemCurrentFile);

                    if (!statsThisPeriod.tblSemsPercentsPerFile.ContainsKey(fileName))
                        statsThisPeriod.tblSemsPercentsPerFile.Add(fileName, new Dictionary<string, double>());
                    statsThisPeriod.tblSemsPercentsPerFile[fileName].Add(element.Key, (double)element.Value / cntNessNodes);


                    //Console.WriteLine("percents added: " + percentsSemCurrentFile.ToString());
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }


            wr.Close();

        }// end processfile
    }

}
